# coding=utf-8
# @Time:2021/4/10
# @Autor:Yhsyao


from bs4 import BeautifulSoup
import urllib.request
import urllib.error
import re
import xlwt

findLink = re.compile(r'<a href="(.*?)">')  # 创建正则表达式对象，电影信息链接
findImg = re.compile(r'<img.*src="(.*?)"', re.S)  # 图片地址
findTitle = re.compile(r'<span class="title">(.*)</span>')  # 片名
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')  # 评分
findJudge = re.compile(r'<span>(\d*)人评价</span>')
findInq = re.compile(r'<span class="inq">(.*)</span>')  # 概况
fingBd = re.compile(r'<p class="">(.*?)</p>', re.S)  # 相关信息


def main():
    baseurl = "https://movie.douban.com/top250?start="
    savePath = r".\\豆瓣电影Top250.xls"
    # 1.爬取数据
    # 2.逐一解析
    datalist = geetData(baseurl)
    # 3.保存数据
    saveData(datalist, savePath)
    print("爬取完成.................")


# 1.爬取数据
def geetData(baseurl):
    datalist = []
    for i in range(10):
        url = baseurl + str(i * 25)
        html = askUrl(url)

        # 2.逐一解析
        soup = BeautifulSoup(html, "html.parser")  # 解析HTML文件
        for item in soup.find_all('div', class_="item"):  # 根据div及class查找
            # print(item)
            data = []  # 保存一部电影所有信息
            item = str(item)

            # 电影信息链接
            link = re.findall(findLink, item)[0]
            data.append(link)

            # 图片地址
            img = re.findall(findImg, item)[0]
            data.append(img)

            # 电影名
            titles = re.findall(findTitle, item)
            if len(titles) == 2:
                ctitle = titles[0]
                data.append(ctitle)  # 第一个中文名
                otitle = titles[1].replace("/", "")
                data.append(otitle)  # 第二个外名
            else:
                data.append(titles[0])
                data.append("")  # 外国名留空

            # 评分
            rating = re.findall(findRating, item)[0]
            data.append(rating)

            # 评价人数
            judge = re.findall(findJudge, item)[0]
            data.append(judge)

            # 电影概况
            inqs = re.findall(findInq, item)
            if len(inqs) != 0:
                inq = inqs[0].replace("。", "")
                data.append(inq)
            else:
                data.append(inqs)

            # 电影相关信息
            bd = re.findall(fingBd, item)[0]
            bd = re.sub(r'<br(\s+)?/>(\s+)?', " ", bd)
            bd = re.sub('/', " ", bd)
            data.append(bd.strip())  # 去空格

            datalist.append(data)
    # print(datalist)
    return datalist


# 得到指定URL网页内容
def askUrl(url):
    # 用户代理，浏览器信息
    head = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
    request = urllib.request.Request(url, headers=head)
    html = ""
    try:
        respone = urllib.request.urlopen(request)
        html = respone.read().decode("utf-8")
        # print(html)
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    return html


def saveData(datalist, savePath):
    # 3.保存数据
    workbook = xlwt.Workbook(encoding="utf-8", style_compression=0)
    sheet = workbook.add_sheet("豆瓣电影Top250", cell_overwrite_ok=True)
    col = ("序号", "电影详情链接", "图片链接", "中文名", "英文名", "评分", "评价数", "概况", "相关信息")

    for i in range(0, 9):
        sheet.write(0, i, col[i])
    for i in range(len(datalist)):
        print("正获取第%d条..." % (i + 1))
        data = datalist[i]
        for j in range(8):
            sheet.write(i+1,0,str(i+1))
            sheet.write(i + 1, j+1, data[j])
    workbook.save(savePath)


if __name__ == '__main__':
    main()
